# -*- coding: utf-8 -*-
import re
from datetime import datetime
import scrapy
from urllib import parse
from scrapy.http import  Request
from ArticleSpider.items import JobboleArticleItem

from ArticleSpider.utils.common import get_md5
from scrapy.loader import ItemLoader

class JobboleSpider(scrapy.Spider):
   
    def __init__(self):
        super(JobboleSpider, self).__init__()
        self.num = 1 
                

    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']

    start_urls = ['http://blog.jobbole.com/all-posts/']

    def parse(self, response):
        '''
        1.获取文章列表中的文章url并且交给scrapy 下载后并进行解析
        2.获取下一页的url并且交给scrapy下载，下载完成后交给parse
        '''
        post_nodes = response.css('#archive .floated-thumb .post-thumb a')
        # print(len(post_nodes))
        for post_node in post_nodes:
            # 交给scrapy下载
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            # print(image_url)
            if not bool(image_url):
                image_url = "NO front-image !"
            # parse.urljoin(response.url, post_url)拼接路径
            yield Request(
                url=parse.urljoin(response.url, post_url),
                meta={"front_image_url": image_url},
                callback=self.parse_detail
            )


        # 提取下一页url并且交给scrapy下载
        # .next.page-numbers两个class之间没有空格，表示当前元素有两个class
        # 有空格就表示有层级关系
        next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_url:

            yield Request(url=parse.urljoin(response.url, next_url), callback=self.parse)

    def parse_detail(self, response):
        if self.num <= 500:
            self.num += 1;
        else:
            return False
        # 定位到标题元素位置[xpath] and [css选择器]
        # title = response.xpath('//*[@id="post-114473"]/div[1]/h1/text()').extract_first()
        title = response.xpath('//*[@class="entry-header"]/h1/text()').extract_first()


        # 定位到发布时间元素位置
        create_date = response.xpath('//*[@class="entry-meta-hide-on-mobile"]/text()').extract_first("").strip().replace("·", '').strip()


        # 获取点赞数
        praise_num = response.xpath('//span[contains(@class, "vote-post-up")]/h10/text()').extract_first("")
        # print("文章点赞量：", praise_num.extract()[0])

        # 获取收藏数
        fav_num = response.xpath('//span[contains(@class, "bookmark-btn")]/text()')
        p = re.match(".*?(\d+).*", fav_num.extract()[0])
        if p:
            fav_num = int(p.group(1))
        else:
            fav_num = 0

        front_image_url = response.meta["front_image_url"]

        content = response.css("div.entry").extract_first("")

        article_item = JobboleArticleItem()

        article_item["title"] = title
        article_item["url"] = response.url
        article_item["url_obj_id"] = get_md5(response.url)
        try:
            create_date = datetime.strptime(create_date, "%Y/%m/%d").date()
        except:
            create_date = datetime.now().date()

        article_item["create_date"] = create_date
        article_item["front_image_url"] = [front_image_url]
        article_item["fav_num"] = fav_num
        article_item["content"] = content
        article_item["front_image_path"] = front_image_url
        article_item["praise_num"] = praise_num

        yield article_item


